
"""
This fetches random page names within the living people subcategory
from erwin85's toolserver and saves them in a JSON file.
"""
# $Id: get_living_people.py 10 2011-07-29 00:51:39Z postshift@gmail.com $

import re
import sys
import json
import urllib
import traceback
from time import time, sleep
from random import random
from subprocess import Popen, PIPE

names = []
desired_num_names = 10000
outputfilename = "names.%d.%.0f.js" % (desired_num_names, time())

num_names = 0
while num_names < desired_num_names:
    # give it a rest of average one second...
    sleep(random() * 2)
    try:
        output = Popen(
            ['wget', '--max-redirect=0', '-O -', 'http://toolserver.org/~erwin85/randomarticle.php?lang=en&family=wikipedia&categories=Living+people&subcats=1&d=2'], 
            stdout=PIPE, stderr=PIPE).communicate()[1]
    except Exception, exc:
        print "# FAILED: " + traceback,format_exc(exc)
        continue

    if not output:
        print "# got a blank output from Popen"
        continue

    try:
        name = re.search(".*title..(?P<name>\S*)", output, re.M).group('name')
        name = urllib.unquote(re.sub("_", " ", name))
    except Exception, exc:
        print "# FAILED: " + traceback,format_exc(exc)
        continue

    if not name:
        print "# got a blank name"
        continue

    names.append(name)
    num_names += 1

    sys.stdout.write(name)
    # flush the output to a file
    fh = open(outputfilename + ".bup", "w")
    fh.write(json.dumps(names))
    fh.flush()
    fh.close()
    sys.stdout.write("\n")

print "# Saving final output..."
# flush the output to a file
fh = open(outputfilename, "w")
fh.write(json.dumps(names))
fh.flush()
fh.close()
print "Done!!!"
